package com.g2room.driving.grap.article.kc;

import java.io.IOException;
import java.net.MalformedURLException;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import com.g2room.driving.model.po.ArticleGrap;
import com.g2room.driving.util.DbUtil;
import com.g2room.driving.util.HtmlUtil;
import com.g2room.driving.util.IpUtil;


public class GrapQczjDataByProxy {
	static Connection connection = null;
	static PreparedStatement  ps = null;
	static Map<String, String> errorUrls = new HashMap<String, String>();
	static List<ArticleGrap> articleList = new ArrayList<ArticleGrap> ();
	
	private static final String baseUrl = "http://car.autohome.com.cn";
	private static final String homePage = "http://car.autohome.com.cn/shuyu/index.html";
	private static final String soruce = "汽车之家";
//	private static Map<String, String> urlMap = new HashMap<String, String>();
//	static {
//		urlMap.put("专业名词", "http://car.autohome.com.cn/shuyu/list_1_0_40.html");
//		urlMap.put("电动车", "http://car.autohome.com.cn/shuyu/list_1_77_76.html");
//		urlMap.put("111", "222");
//		urlMap.put("111", "222");
//		urlMap.put("111", "222");
//		urlMap.put("111", "222");
//		urlMap.put("111", "222");
//		urlMap.put("111", "222");
//	}
	private static Connection getConn() throws ClassNotFoundException, SQLException {
		if(null == connection) {
			connection = DbUtil.getConn();
		}
		return connection;
	}
	static Map<String, String> errMap = new HashMap<String, String>();
	public static void main(String[] args) throws Exception{
		GrapQczjDataByProxy gkd = new GrapQczjDataByProxy();
		gkd.getNav();
		// 最后输出失败的链接到数据库
		insertError2Db();
    }
	
	public static Document getDoc(String url) {
		String ip = IpUtil.getIp();
		Document doc = null;
		try {
			doc =Jsoup.connect(url)
//				  .data("query", "Java")   // 请求参数
					.header("X-Real-IP", ip)
					.header("x-forwarded-for", ip)
					.header("WL-Proxy-Client-IP", ip)
					.userAgent("Mozilla/5.0 (Windows NT 5.1; rv:6.0.2) Gecko/20100101 Firefox/6.0.2") // 设置 User-Agent 
//				.cookie("auth", "token") // 设置 cookie 
					.timeout(5000)           // 设置连接超时时间
					.get(); // get方式访问
		} catch (IOException e) {
			e.printStackTrace();
		}
		return doc;
	}
	
	
	/**
	 * 获取头部导航菜单
	 * @param url
	 * @throws FailingHttpStatusCodeException
	 * @throws MalformedURLException
	 * @throws IOException
	 */
	public void getNav() {
		Element el = getDoc(homePage).getElementById("frame_tree");
		Elements links = el.getElementsByTag("a");
		Map<String, String> pages = new HashMap<String, String>();
		for (Element link : links) {
			pages.put(baseUrl + link.attr("href"), link.text());
		}
        
        for(String key : pages.keySet()) {
        	if(key.equals("全部")) continue;
	        try {
	        	Thread.sleep(IpUtil.genRandomNum() * 500);
	        } catch (InterruptedException e) {
	        	e.printStackTrace();
	        }
	        getPage(key, pages.get(key));
        }
	}
	
	/**
	 * 获取分页数据
	 * @param url
	 * @throws FailingHttpStatusCodeException
	 * @throws MalformedURLException
	 * @throws IOException
	 */
	public void getPage(String url, String categoryName) {
		System.out.println("start get page data!" + categoryName);
		String id = url.substring(url.lastIndexOf("_") + 1, url.lastIndexOf("."));
		Elements dds = getDoc(url).getElementById("s" + id).child(0).children();
		Map<String, String> details = new HashMap<String, String>();
		String typeName = "";
		// 遍历获取下面的子节点
		for(Element el : dds) {
			String tag = el.tagName();
			if(tag.equalsIgnoreCase("dt")) {
				typeName = el.text();
			} else if (tag.equalsIgnoreCase("dd")) {
				details.put(el.child(0).attr("href"), typeName);
			}
		}
        
        for(String href : details.keySet()) {
        	try {
				Thread.sleep(IpUtil.genRandomNum() * 500);
			} catch (InterruptedException e) {
				e.printStackTrace();
			}
        	getDetail(baseUrl + href, categoryName, details.get(href));
        }
	}
	
	/**
	 * 获取列表数据
	 * @param url
	 * @throws FailingHttpStatusCodeException
	 * @throws MalformedURLException
	 * @throws IOException
	 */
	public void getDetail(String url, String categoryName, String typeName) {
		System.out.println("categoryName==" + categoryName + "   ,typeName=" + typeName +  ",start get list data!url==  " + url);
		try {
			ArticleGrap arti = new  ArticleGrap();
			arti.setCategoryName(categoryName);
			arti.setTypeName(typeName);
			arti.setSource(soruce);
			arti.setSourceUrl(url);
			Document doc = getDoc(url);
			Element content = doc.getElementsByAttributeValue("class", "conleft").get(0);
			String cont = content.html();
			cont = HtmlUtil.delATag(cont);
			cont = HtmlUtil.delScriptAndStyleTag(cont);
			cont = HtmlUtil.delNoteTag(cont);
			arti.setTitle(doc.getElementById("lblName").text());
			// 删除a标签
			arti.setContent(cont);
			
			insert2Db(arti);
		} catch (Exception e) {
			errMap.put(url, typeName);
			e.printStackTrace();
		}
	}
	
	int sqlCount = 0;
	public void insert2Db(ArticleGrap articles) {
		try {
			if(sqlCount == 0) ps = getConn().prepareStatement("insert into d_article_grap(title,content,type_name,category_name,source,source_url, state) values(?,?,?,?,?,?,?)");
			// title,content,publish_time,type_name,source,source_url
			ps.setString(1, articles.getTitle());
			ps.setString(2, articles.getContent());
			ps.setString(3, articles.getTypeName());
			ps.setString(4, articles.getCategoryName());
			ps.setString(5, soruce);
			ps.setString(6, articles.getSourceUrl());
			ps.setInt(7, 9);
			ps.execute();
		} catch(Exception e) {
			e.printStackTrace();
		}
	}
	
	public static void insertError2Db() {
		try {
			PreparedStatement  ps = getConn().prepareStatement("insert into d_article_grap_err(title,source,source_url,type_name) values(?,?,?,?)");
			for(String url : errorUrls.keySet()) {
				ps.setString(1, errorUrls.get(url));
				ps.setString(2, soruce);
				ps.setString(3, url);
				ps.setString(4, "导航和列表错误");
				ps.addBatch();
			}
			for(ArticleGrap article : articleList) {
				ps.setString(1, article.getTitle());
				ps.setString(2, soruce);
				ps.setString(3, article.getSourceUrl());
				ps.setString(4, article.getTypeName());
				ps.addBatch();
			}
			ps.executeBatch();
			if(null != ps) ps.close();
			if(null != connection) connection.close();
		} catch(Exception e) {
			e.printStackTrace();
		}
	}
}
